﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;

namespace SimpleCrawler {
  class SimpleCrawler {
    private Hashtable urls = new Hashtable();
    private int count = 0;
    static void Main(string[] args) {
      
      SimpleCrawler myCrawler = new SimpleCrawler();
      string startUrl = "https://www.baidu.com/";
      myCrawler.urls.Add(startUrl, false);//加入初始页面
      new Thread(myCrawler.Crawl).Start();
    }

    private void Crawl() {
      Console.WriteLine("开始爬行了.... ");
      while (true) {
        string current = null;
        foreach (string url in urls.Keys) {
          if ((bool)urls[url]) 
                continue;  //已经下载过就不再下载，即value为true
          current = url;
          
        }

        if (current == null || count > 10) break;
        Console.WriteLine("爬行" + current + "页面!");
        string html = DownLoad(current); // 下载
        urls[current] = true;
        count++;
        Parse(html);//解析,并加入新的链接
        Console.WriteLine("爬行结束");
      }
    }

    public string DownLoad(string url) {
      try {
        WebClient webClient = new WebClient();
        webClient.Encoding = Encoding.UTF8;
        string html = webClient.DownloadString(url);
        string fileName = @"D:\Result\"+count.ToString();
        string fileName1 = @"D:\zyl\" + count.ToString();
        File.WriteAllText(fileName, html, Encoding.UTF8);
        File.WriteAllText(fileName1, url, Encoding.UTF8);
        return html;
      }
      catch (Exception ex) {
        Console.WriteLine(ex.Message);
        return "";
      }
    }

    private void Parse(string html) {
                string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
                MatchCollection matches = new Regex(strRef).Matches(html);
                foreach (Match match in matches) {
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1)
                            .Trim('"', '\"', '#', '>');
                if (strRef.Length == 0){
                            continue;
                        }
                try
                {
                    Uri uri = new Uri(strRef);
                    if (!uri.IsAbsoluteUri)
                    {   //直接判断是否为绝对地址....具体如何转换暂未解决
                        continue;
                    }
                }catch(Exception e)
                {
                    continue;
                }
                if(!new Uri(strRef).IsAbsoluteUri)
                {
                    continue;
                }
         
                    
        
            if (urls[strRef] == null&&Choose(strRef)) 
                    urls[strRef] = false;
      }
    }
   private bool Choose(String url) {  //判断是否为html/htm/aspx/jsp网页
            Regex regex = new Regex("(.html|.htm|.aspx|.jsp)");
            if (regex.Matches(url).Count == 0)
                return true;
            else
                return false;

        }
  }
}
